package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

//const targetUrl = "http://www.zhenai.com/zhenghun"
const userAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36"

var rateLimit = time.Tick(10 * time.Millisecond);

func Fetcher(url string) ([]byte, error) {
	<-rateLimit
	request, err := http.NewRequest("GET", url, nil)

	if err != nil {
		log.Fatal(err)
		return nil, err
	}
	request.Header.Set("User-Agent", userAgent)

	client := http.Client{}
	response, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	defer response.Body.Close()

	//返回的状态码
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("wrong status code:%d", response.StatusCode)
	}
	bodyReader := bufio.NewReader(response.Body)
	e := determineEncoding(bodyReader)
	reader := transform.NewReader(bodyReader, e.NewDecoder())
	return ioutil.ReadAll(reader)
}

//增加转码
func determineEncoding(reader *bufio.Reader) encoding.Encoding {

	//使用peek读取数据
	bytes, err := reader.Peek(1024)

	if err != nil {
		log.Printf("Fetcher error:%v", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
